import requests
from bs4 import BeautifulSoup


class WebSpider(object):
    def __init__(self):
        self.url = 'https://www.ted.com/talks'
        self.talks = []
        self.links = []

    def get_talks_links(self):
        # response = requests.get(self.url)
        # html_text = response.text
        # read file from local
        html_text = open('ted.html')
        bs = BeautifulSoup(html_text, 'html.parser')
        browser_results_div = bs.find('div', id='browse-results')
        browser_results_div_h4 = browser_results_div.find_all('h4', class_='f-w:700 h9 m5')
        for item in browser_results_div_h4:
            self.talks.append(item.find('a').string)
            self.links.append(item.find('a').get('href')+'\n')

    def write_file(self, path, text):
        with open(path, 'a', encoding='UTF-8') as file:
            file.writelines(text)


if __name__ == '__main__':
    spider = WebSpider()
    spider.get_talks_links()
    spider.write_file('ted_talks.txt', spider.talks)